{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Char LSTM Example.\n",
    "This example aims to show how to use lstm to build a char level language model, and generate text from it. \n",
    "We use a tiny shakespeare text for demo purpose. \n",
    "\n",
    "Data can be found at [https://github.com/dmlc/web-data/tree/master/mxnet/tinyshakespeare](https://github.com/dmlc/web-data/tree/master/mxnet/tinyshakespeare). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from collections import defaultdict\n",
    "import mxnet as mx\n",
    "import numpy as np\n",
    "import sys\n",
    "import lstm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set basic network parameters. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "seq_len = 32\n",
    "num_hidden = 256\n",
    "num_embed = 256\n",
    "num_lstm_layer = 2\n",
    "num_round = 21\n",
    "learning_rate= 1\n",
    "wd=0.00001\n",
    "momentum=0.0\n",
    "clip_gradient=1\n",
    "update_period = 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make dictionary from text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_dict(text, max_vocab=10000):\n",
    "    lst = list(text)\n",
    "    cnt = Counter(lst)\n",
    "    print(\"Total unique char: %d\" % len(cnt))\n",
    "    common = cnt.most_common(max_vocab - 1)\n",
    "    dic = defaultdict(int)\n",
    "    idx = 0\n",
    "    for c, _ in common:\n",
    "        dic[c] = idx\n",
    "        idx += 1\n",
    "    if len(dic) == max_vocab - 1:\n",
    "        dic[\"_UNKNOWN_\"] = idx\n",
    "    return dic\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Transfer text into data batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def make_batch(file_path, batch_size=32, seq_lenth=32, max_vocab=10000, dic=None):\n",
    "    fi = open(file_path)\n",
    "    text = fi.read()\n",
    "    fi.close()\n",
    "    if dic == None:\n",
    "        dic = make_dict(text, max_vocab)\n",
    "    lookup_table = dict((idx, c) for c, idx in dic.items())\n",
    "    char_lst = list(text)\n",
    "    num_batch = int(len(char_lst) / batch_size)\n",
    "    char_lst = char_lst[:num_batch * batch_size]\n",
    "    data = np.zeros((num_batch, batch_size), dtype=\"float32\")\n",
    "    idx = 0\n",
    "    for j in range(batch_size):\n",
    "        for i in range(num_batch):\n",
    "            if char_lst[idx] in dic:\n",
    "                data[i][j] = dic[char_lst[idx]]\n",
    "            else:\n",
    "                char_lst[idx] = dic[\"_UNKNOWN_\"]\n",
    "            idx += 1\n",
    "    return data, dic, lookup_table\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total unique char: 65\n"
     ]
    }
   ],
   "source": [
    "X, dic, lookup_table = make_batch(\"./input.txt\", batch_size=batch_size, seq_lenth=seq_len)\n",
    "vocab = len(dic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Move tail text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def drop_tail(X, seq_len):\n",
    "    shape = X.shape\n",
    "    nstep = int(shape[0] / seq_len)\n",
    "    return X[0:(nstep * seq_len), :]\n",
    "\n",
    "train_val_fraction = 0.9\n",
    "size = X.shape[0]\n",
    "X_train = X[:int(size * train_val_fraction), :]\n",
    "X_val = X[int(size * train_val_fraction):, :]\n",
    "\n",
    "X_train = drop_tail(X_train, seq_len)\n",
    "X_val = drop_tail(X_val, seq_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set up LSTM model on GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = lstm.setup_rnn_model(mx.gpu(),\n",
    "                             num_lstm_layer=num_lstm_layer,\n",
    "                             seq_len=seq_len,\n",
    "                             num_hidden=num_hidden,\n",
    "                             num_embed=num_embed,\n",
    "                             num_label=vocab,\n",
    "                             batch_size=batch_size,\n",
    "                             input_size=vocab,\n",
    "                             initializer=mx.initializer.Uniform(0.1),\n",
    "                             dropout=0.)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training swith train.shape=(31360, 32)\n",
      "Training swith val.shape=(3456, 32)\n",
      "batch_size=32\n",
      "seq_len=32\n",
      "Epoch [125] Train: NLL=3.368, Prep=29.019\n",
      "Epoch [250] Train: NLL=3.289, Prep=26.811\n",
      "Epoch [375] Train: NLL=3.180, Prep=24.044\n",
      "Epoch [500] Train: NLL=3.070, Prep=21.534\n",
      "Epoch [625] Train: NLL=2.971, Prep=19.503\n",
      "Epoch [750] Train: NLL=2.891, Prep=18.011\n",
      "Epoch [875] Train: NLL=2.824, Prep=16.846\n",
      "Iter [0] Train: Time: 40.182 sec, NLL=2.775, Prep=16.041\n",
      "Iter [0] Val: NLL=2.288, Prep=9.857\n",
      "Epoch [1000] Train: NLL=2.347, Prep=10.451\n",
      "Epoch [1125] Train: NLL=2.321, Prep=10.188\n",
      "Epoch [1250] Train: NLL=2.298, Prep=9.951\n",
      "Epoch [1375] Train: NLL=2.276, Prep=9.741\n",
      "Epoch [1500] Train: NLL=2.256, Prep=9.541\n",
      "Epoch [1625] Train: NLL=2.234, Prep=9.338\n",
      "Epoch [1750] Train: NLL=2.215, Prep=9.160\n",
      "Epoch [1875] Train: NLL=2.196, Prep=8.987\n",
      "Iter [1] Train: Time: 40.342 sec, NLL=2.184, Prep=8.885\n",
      "Iter [1] Val: NLL=1.988, Prep=7.298\n",
      "Epoch [2000] Train: NLL=2.050, Prep=7.766\n",
      "Epoch [2125] Train: NLL=2.032, Prep=7.631\n",
      "Epoch [2250] Train: NLL=2.014, Prep=7.490\n",
      "Epoch [2375] Train: NLL=2.002, Prep=7.405\n",
      "Epoch [2500] Train: NLL=1.988, Prep=7.297\n",
      "Epoch [2625] Train: NLL=1.974, Prep=7.200\n",
      "Epoch [2750] Train: NLL=1.961, Prep=7.106\n",
      "Epoch [2875] Train: NLL=1.949, Prep=7.024\n",
      "Iter [2] Train: Time: 40.377 sec, NLL=1.943, Prep=6.981\n",
      "Iter [2] Val: NLL=1.808, Prep=6.101\n",
      "Reset learning rate to 0.9\n",
      "Epoch [3000] Train: NLL=1.850, Prep=6.359\n",
      "Epoch [3125] Train: NLL=1.844, Prep=6.323\n",
      "Epoch [3250] Train: NLL=1.831, Prep=6.238\n",
      "Epoch [3375] Train: NLL=1.822, Prep=6.185\n",
      "Epoch [3500] Train: NLL=1.812, Prep=6.124\n",
      "Epoch [3625] Train: NLL=1.805, Prep=6.077\n",
      "Epoch [3750] Train: NLL=1.797, Prep=6.033\n",
      "Epoch [3875] Train: NLL=1.790, Prep=5.990\n",
      "Iter [3] Train: Time: 40.348 sec, NLL=1.787, Prep=5.973\n",
      "Iter [3] Val: NLL=1.695, Prep=5.446\n",
      "Epoch [4000] Train: NLL=1.736, Prep=5.676\n",
      "Epoch [4125] Train: NLL=1.734, Prep=5.663\n",
      "Epoch [4250] Train: NLL=1.722, Prep=5.595\n",
      "Epoch [4375] Train: NLL=1.715, Prep=5.555\n",
      "Epoch [4500] Train: NLL=1.707, Prep=5.514\n",
      "Epoch [4625] Train: NLL=1.703, Prep=5.492\n",
      "Epoch [4750] Train: NLL=1.697, Prep=5.459\n",
      "Epoch [4875] Train: NLL=1.693, Prep=5.434\n",
      "Iter [4] Train: Time: 40.372 sec, NLL=1.691, Prep=5.427\n",
      "Iter [4] Val: NLL=1.617, Prep=5.039\n",
      "Epoch [5000] Train: NLL=1.659, Prep=5.257\n",
      "Epoch [5125] Train: NLL=1.653, Prep=5.221\n",
      "Epoch [5250] Train: NLL=1.645, Prep=5.179\n",
      "Epoch [5375] Train: NLL=1.638, Prep=5.143\n",
      "Epoch [5500] Train: NLL=1.633, Prep=5.119\n",
      "Epoch [5625] Train: NLL=1.629, Prep=5.101\n",
      "Epoch [5750] Train: NLL=1.625, Prep=5.079\n",
      "Epoch [5875] Train: NLL=1.621, Prep=5.059\n",
      "Iter [5] Train: Time: 40.363 sec, NLL=1.621, Prep=5.059\n",
      "Iter [5] Val: NLL=1.569, Prep=4.804\n",
      "Reset learning rate to 0.81\n",
      "Epoch [6000] Train: NLL=1.603, Prep=4.966\n",
      "Epoch [6125] Train: NLL=1.588, Prep=4.895\n",
      "Epoch [6250] Train: NLL=1.585, Prep=4.879\n",
      "Epoch [6375] Train: NLL=1.579, Prep=4.852\n",
      "Epoch [6500] Train: NLL=1.574, Prep=4.827\n",
      "Epoch [6625] Train: NLL=1.571, Prep=4.812\n",
      "Epoch [6750] Train: NLL=1.567, Prep=4.793\n",
      "Iter [6] Train: Time: 40.353 sec, NLL=1.565, Prep=4.781\n",
      "Iter [6] Val: NLL=1.529, Prep=4.615\n",
      "Epoch [6875] Train: NLL=1.574, Prep=4.824\n",
      "Epoch [7000] Train: NLL=1.560, Prep=4.760\n",
      "Epoch [7125] Train: NLL=1.545, Prep=4.686\n",
      "Epoch [7250] Train: NLL=1.544, Prep=4.684\n",
      "Epoch [7375] Train: NLL=1.538, Prep=4.654\n",
      "Epoch [7500] Train: NLL=1.534, Prep=4.635\n",
      "Epoch [7625] Train: NLL=1.530, Prep=4.620\n",
      "Epoch [7750] Train: NLL=1.528, Prep=4.607\n",
      "Iter [7] Train: Time: 40.353 sec, NLL=1.526, Prep=4.598\n",
      "Iter [7] Val: NLL=1.496, Prep=4.463\n",
      "Epoch [7875] Train: NLL=1.530, Prep=4.619\n",
      "Epoch [8000] Train: NLL=1.522, Prep=4.579\n",
      "Epoch [8125] Train: NLL=1.511, Prep=4.533\n",
      "Epoch [8250] Train: NLL=1.511, Prep=4.532\n",
      "Epoch [8375] Train: NLL=1.506, Prep=4.508\n",
      "Epoch [8500] Train: NLL=1.503, Prep=4.494\n",
      "Epoch [8625] Train: NLL=1.499, Prep=4.479\n",
      "Epoch [8750] Train: NLL=1.497, Prep=4.467\n",
      "Iter [8] Train: Time: 40.371 sec, NLL=1.495, Prep=4.461\n",
      "Iter [8] Val: NLL=1.481, Prep=4.396\n",
      "Reset learning rate to 0.729\n",
      "Epoch [8875] Train: NLL=1.478, Prep=4.384\n",
      "Epoch [9000] Train: NLL=1.489, Prep=4.434\n",
      "Epoch [9125] Train: NLL=1.482, Prep=4.400\n",
      "Epoch [9250] Train: NLL=1.480, Prep=4.391\n",
      "Epoch [9375] Train: NLL=1.474, Prep=4.368\n",
      "Epoch [9500] Train: NLL=1.471, Prep=4.355\n",
      "Epoch [9625] Train: NLL=1.469, Prep=4.343\n",
      "Epoch [9750] Train: NLL=1.466, Prep=4.333\n",
      "Iter [9] Train: Time: 40.344 sec, NLL=1.465, Prep=4.329\n",
      "Iter [9] Val: NLL=1.453, Prep=4.278\n",
      "Epoch [9875] Train: NLL=1.458, Prep=4.297\n",
      "Epoch [10000] Train: NLL=1.466, Prep=4.331\n",
      "Epoch [10125] Train: NLL=1.460, Prep=4.305\n",
      "Epoch [10250] Train: NLL=1.456, Prep=4.289\n",
      "Epoch [10375] Train: NLL=1.452, Prep=4.270\n",
      "Epoch [10500] Train: NLL=1.449, Prep=4.260\n",
      "Epoch [10625] Train: NLL=1.447, Prep=4.248\n",
      "Epoch [10750] Train: NLL=1.445, Prep=4.242\n",
      "Iter [10] Train: Time: 40.341 sec, NLL=1.444, Prep=4.240\n",
      "Iter [10] Val: NLL=1.438, Prep=4.211\n",
      "Epoch [10875] Train: NLL=1.447, Prep=4.250\n",
      "Epoch [11000] Train: NLL=1.445, Prep=4.243\n",
      "Epoch [11125] Train: NLL=1.440, Prep=4.222\n",
      "Epoch [11250] Train: NLL=1.436, Prep=4.205\n",
      "Epoch [11375] Train: NLL=1.434, Prep=4.196\n",
      "Epoch [11500] Train: NLL=1.432, Prep=4.185\n",
      "Epoch [11625] Train: NLL=1.429, Prep=4.175\n",
      "Epoch [11750] Train: NLL=1.428, Prep=4.169\n",
      "Iter [11] Train: Time: 40.352 sec, NLL=1.427, Prep=4.168\n",
      "Iter [11] Val: NLL=1.429, Prep=4.174\n",
      "Reset learning rate to 0.6561\n",
      "Epoch [11875] Train: NLL=1.431, Prep=4.182\n",
      "Epoch [12000] Train: NLL=1.424, Prep=4.154\n",
      "Epoch [12125] Train: NLL=1.422, Prep=4.145\n",
      "Epoch [12250] Train: NLL=1.418, Prep=4.127\n",
      "Epoch [12375] Train: NLL=1.414, Prep=4.113\n",
      "Epoch [12500] Train: NLL=1.412, Prep=4.105\n",
      "Epoch [12625] Train: NLL=1.410, Prep=4.096\n",
      "Iter [12] Train: Time: 40.357 sec, NLL=1.409, Prep=4.091\n",
      "Iter [12] Val: NLL=1.417, Prep=4.124\n",
      "Epoch [12750] Train: NLL=1.435, Prep=4.201\n",
      "Epoch [12875] Train: NLL=1.417, Prep=4.123\n",
      "Epoch [13000] Train: NLL=1.408, Prep=4.086\n",
      "Epoch [13125] Train: NLL=1.409, Prep=4.091\n",
      "Epoch [13250] Train: NLL=1.404, Prep=4.073\n",
      "Epoch [13375] Train: NLL=1.401, Prep=4.058\n",
      "Epoch [13500] Train: NLL=1.398, Prep=4.048\n",
      "Epoch [13625] Train: NLL=1.397, Prep=4.041\n",
      "Iter [13] Train: Time: 40.356 sec, NLL=1.396, Prep=4.038\n",
      "Iter [13] Val: NLL=1.411, Prep=4.102\n",
      "Epoch [13750] Train: NLL=1.414, Prep=4.114\n",
      "Epoch [13875] Train: NLL=1.402, Prep=4.063\n",
      "Epoch [14000] Train: NLL=1.395, Prep=4.036\n",
      "Epoch [14125] Train: NLL=1.396, Prep=4.037\n",
      "Epoch [14250] Train: NLL=1.392, Prep=4.023\n",
      "Epoch [14375] Train: NLL=1.389, Prep=4.010\n",
      "Epoch [14500] Train: NLL=1.386, Prep=4.000\n",
      "Epoch [14625] Train: NLL=1.385, Prep=3.995\n",
      "Iter [14] Train: Time: 40.344 sec, NLL=1.384, Prep=3.992\n",
      "Iter [14] Val: NLL=1.400, Prep=4.055\n",
      "Reset learning rate to 0.59049\n",
      "Epoch [14750] Train: NLL=1.378, Prep=3.966\n",
      "Epoch [14875] Train: NLL=1.390, Prep=4.014\n",
      "Epoch [15000] Train: NLL=1.383, Prep=3.986\n",
      "Epoch [15125] Train: NLL=1.382, Prep=3.982\n",
      "Epoch [15250] Train: NLL=1.377, Prep=3.965\n",
      "Epoch [15375] Train: NLL=1.375, Prep=3.957\n",
      "Epoch [15500] Train: NLL=1.372, Prep=3.945\n",
      "Epoch [15625] Train: NLL=1.371, Prep=3.938\n",
      "Iter [15] Train: Time: 40.352 sec, NLL=1.370, Prep=3.936\n",
      "Iter [15] Val: NLL=1.393, Prep=4.026\n",
      "Epoch [15750] Train: NLL=1.368, Prep=3.927\n",
      "Epoch [15875] Train: NLL=1.380, Prep=3.974\n",
      "Epoch [16000] Train: NLL=1.374, Prep=3.951\n",
      "Epoch [16125] Train: NLL=1.371, Prep=3.940\n",
      "Epoch [16250] Train: NLL=1.367, Prep=3.922\n",
      "Epoch [16375] Train: NLL=1.364, Prep=3.912\n",
      "Epoch [16500] Train: NLL=1.362, Prep=3.905\n",
      "Epoch [16625] Train: NLL=1.361, Prep=3.900\n",
      "Iter [16] Train: Time: 40.358 sec, NLL=1.360, Prep=3.898\n",
      "Iter [16] Val: NLL=1.389, Prep=4.012\n",
      "Epoch [16750] Train: NLL=1.367, Prep=3.924\n",
      "Epoch [16875] Train: NLL=1.367, Prep=3.923\n",
      "Epoch [17000] Train: NLL=1.363, Prep=3.907\n",
      "Epoch [17125] Train: NLL=1.360, Prep=3.895\n",
      "Epoch [17250] Train: NLL=1.357, Prep=3.886\n",
      "Epoch [17375] Train: NLL=1.355, Prep=3.878\n",
      "Epoch [17500] Train: NLL=1.353, Prep=3.867\n",
      "Epoch [17625] Train: NLL=1.352, Prep=3.864\n",
      "Iter [17] Train: Time: 40.347 sec, NLL=1.352, Prep=3.864\n",
      "Iter [17] Val: NLL=1.384, Prep=3.990\n",
      "Reset learning rate to 0.531441\n",
      "Epoch [17750] Train: NLL=1.362, Prep=3.903\n",
      "Epoch [17875] Train: NLL=1.355, Prep=3.877\n",
      "Epoch [18000] Train: NLL=1.353, Prep=3.870\n",
      "Epoch [18125] Train: NLL=1.348, Prep=3.851\n",
      "Epoch [18250] Train: NLL=1.346, Prep=3.843\n",
      "Epoch [18375] Train: NLL=1.344, Prep=3.834\n",
      "Epoch [18500] Train: NLL=1.342, Prep=3.827\n",
      "Iter [18] Train: Time: 40.354 sec, NLL=1.341, Prep=3.823\n",
      "Iter [18] Val: NLL=1.378, Prep=3.967\n",
      "Epoch [18625] Train: NLL=1.370, Prep=3.935\n",
      "Epoch [18750] Train: NLL=1.352, Prep=3.863\n",
      "Epoch [18875] Train: NLL=1.345, Prep=3.838\n",
      "Epoch [19000] Train: NLL=1.346, Prep=3.841\n",
      "Epoch [19125] Train: NLL=1.341, Prep=3.823\n",
      "Epoch [19250] Train: NLL=1.338, Prep=3.811\n",
      "Epoch [19375] Train: NLL=1.336, Prep=3.803\n",
      "Epoch [19500] Train: NLL=1.334, Prep=3.797\n",
      "Iter [19] Train: Time: 40.370 sec, NLL=1.334, Prep=3.795\n",
      "Iter [19] Val: NLL=1.377, Prep=3.961\n",
      "Epoch [19625] Train: NLL=1.354, Prep=3.874\n",
      "Epoch [19750] Train: NLL=1.344, Prep=3.836\n",
      "Epoch [19875] Train: NLL=1.338, Prep=3.811\n",
      "Epoch [20000] Train: NLL=1.338, Prep=3.813\n",
      "Epoch [20125] Train: NLL=1.334, Prep=3.797\n",
      "Epoch [20250] Train: NLL=1.331, Prep=3.786\n",
      "Epoch [20375] Train: NLL=1.329, Prep=3.778\n",
      "Epoch [20500] Train: NLL=1.328, Prep=3.774\n",
      "Iter [20] Train: Time: 40.363 sec, NLL=1.327, Prep=3.771\n",
      "Iter [20] Val: NLL=1.373, Prep=3.946\n",
      "Reset learning rate to 0.478297\n"
     ]
    }
   ],
   "source": [
    "lstm.train_lstm(model, X_train, X_val,\n",
    "                num_round=num_round,\n",
    "                half_life=3,\n",
    "                update_period=update_period,\n",
    "                learning_rate=learning_rate,\n",
    "                wd=wd,\n",
    "                momentum=momentum,\n",
    "                clip_gradient=clip_gradient)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get parameter from model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "args = dict([(name, arr) for i, arr, grad_arr, name in model.param_blocks])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make a sampler use the parameter we trained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
     ]
    }
   ],
   "source": [
    "batch_size = 1\n",
    "sampler = lstm.setup_rnn_sample_model(mx.cpu(), args, num_lstm_layer, num_hidden, num_embed, vocab, batch_size, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "start = 'a'\n",
    "seq_len = 75\n",
    "X_input_batch = np.zeros((1,1), dtype=\"float32\")\n",
    "X_input_batch[0][0] = dic[start]\n",
    "out = lstm.sample_lstm(sampler, X_input_batch, seq_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lookup predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "an'd and dear victories at sound before.\n",
      "Sir! palient, made me; let it kiss \n"
     ]
    }
   ],
   "source": [
    "chars = [lookup_table[int(out[i][0])] for i in range(seq_len)]\n",
    "print(start + \"\".join(chars))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
